# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import hashlib
import time

import redis
from scrapy.utils.python import to_bytes

from pymysql import *
from w3lib.url import canonicalize_url

from zhaonuo.settings import mysql_conf, redis_conf

class ZhaonuoPipeline(object):
    def open_spider(self, spider):
        self.mysql_conf = mysql_conf
        # 连接数据库
        self.conn = connect(
            host=self.mysql_conf.get('host'),
            port=self.mysql_conf.get('port'),
            database=self.mysql_conf.get('db'),
            user=self.mysql_conf.get('user'),
            password=self.mysql_conf.get('passwd'),
            charset=self.mysql_conf.get('charset'),
        )
        self.cs = self.conn.cursor()

        self.redis_conf = redis_conf
        self.red = redis.StrictRedis(host=self.redis_conf.get('host'),
                                     port=self.redis_conf.get('port'),
                                     db=self.redis_conf.get('db'),
                                     password=self.redis_conf.get('passwd'))

    def close_spider(self, spider):
        # 关闭爬虫前删除反扒指纹
        # err_urls = spider.err_urls
        # name = spider.name
        # name = '{}:dupefilter'.format(name)
        # for url in err_urls:
        #     self.del_fingerprint(url,name)
        self.cs.close()
        self.conn.close()

    def del_fingerprint(self, url, name):
        """请求失败,删除redis中的指纹"""
        fp = hashlib.sha1()
        fp.update(to_bytes('GET'))
        fp.update(to_bytes(canonicalize_url(url)))
        fp.update(b'')
        #print(fp.hexdigest())
        # 这里存的是本爬虫的url指纹集合
        self.red.srem(name, fp.hexdigest())

    def sadd_fingerprint(self, url, name):
        """添加指纹redis中"""
        #print(url)
        fp = hashlib.sha1()
        fp.update(to_bytes('GET'))
        fp.update(to_bytes(canonicalize_url(url)))
        fp.update(b'')
        # print(fp.hexdigest())
        # 这里存的是本爬虫的url指纹集合
        res = self.red.sadd(name, fp.hexdigest())
        return res

    def process_item(self, item, spider):
        #print(spider.name)
        if spider.name =='news_url_new':
            url = item["url"]
            res = self.sadd_fingerprint(url,'news_url')
            # print(res)
            if res:
                from_site = item["n"]
                sql = 'insert ignore into bx_news_url (url, from_site) values(%s, %s)'
                self.conn.ping(reconnect=True)
                rt = self.cs.execute(sql, (url, from_site))
                self.conn.commit()
                # print("*"*50)
                #print("mysql插入成功！")
            else:
                print("url已存在")
        elif spider.name == 'news_detail':
            if item["type"] in [1, 2]:
                see_num = item.get("see_num")
                if not see_num:
                    see_num = ''
                insert_sql = 'INSERT ignore INTO bx_news (title, writer, `from`, addtime, content, abstract, keywords, description,see_num) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                # 还要变更url表中的status
                update_sql = 'UPDATE bx_news_url SET status=1 WHERE uid=%s'
                self.conn.ping(reconnect=True)
                ret1 = self.cs.execute(insert_sql, [item['title'], item['writer'], item['url'], int(time.time()), item['content'],
                                                    item['abstract'], item['keywords'], item['description'], see_num])
                self.conn.ping(reconnect=True)
                ret2 = self.cs.execute(update_sql, item['uid'])  # 如果uid在数据库中不存在，这里返回0条受到影响，应当使详情页可以记录进数据库
                # print(ret1,ret2)
            # 上述两步数据库报错a，会直接记录日志，走不到下边的代码，因此也就不存在提交的可能
            elif item["type"] == 3:
                insert_sql = 'INSERT ignore INTO bx_caiji_asks (ask_title, ask_content, ans_list, addtime, from_url, from_type) VALUES (%s,%s,%s,%s,%s,%s)'
                # 还要变更url表中的status
                update_sql = 'UPDATE bx_news_url SET status=1 WHERE uid=%s'
                self.conn.ping(reconnect=True)
                ret1 = self.cs.execute(insert_sql, [item['ask_title'], item['ask_content'], item['ans_list'],
                                                    int(time.time()), item['from_url'], item['from_type']])
                self.conn.ping(reconnect=True)
                ret2 = self.cs.execute(update_sql, item['uid'])  # 如果uid在数据库中不存在，这里返回0条受到影响，应当使详情页可以记录进数据库

            if ret1 and ret2 is not None:
                self.conn.commit()
                print('提交成功')
            else:
                self.conn.rollback()
                print('提交失败，回滚')
